#!/usr/bin/perl
use strict;
use warnings;

## get options

use Getopt::Long;
my %param = (
    source           => undef,
    target           => undef,
    output_directory => undef,
    n_words          => undef,
    n_select         => undef,
    );
GetOptions(
    "source=s"           => \$param{source},
    "target=s"           => \$param{target},
    "output_directory=s" => \$param{output_directory},
    "n_words=i"          => \$param{n_words},
    "n_select=i"         => \$param{n_select},
    );
while(my ($param, $value) = each(%param)){
    die "$param undefined!" if !defined($value);
}

## for each bitext pair, output to directory ..

open(SOURCE, "<$param{source}") or die "could not open source file!";
binmode(SOURCE, ":utf8");
open(TARGET, "<$param{target}") or die "could not open target file!";
binmode(TARGET, ":utf8");

my $i = 0;
my $n_selected = 0;
while(defined(my $source = <SOURCE>)){
    chomp $source;
    $source =~ s/^\s+(\S|$)/$1/;
    $source =~ s/(^|\S)\s+$/$1/;
    my @source_words = split(/\s+/, $source);

    defined(my $target = <TARGET>) or die "lopsided bitext!";
    chomp $target;
    $target =~ s/^\s+(\S|$)/$1/;
    $target =~ s/(^|\S)\s+$/$1/;
    my @target_words = split(/\s+/, $target);

    ## only choose for hand alignment is both target and source are shorter than $n_words words
    if((scalar(@source_words) < $param{n_words}) &&
       (scalar(@target_words) < $param{n_words})){
	open(OUTPUT, ">/tmp/a") or die "could not open output file!";
	binmode(OUTPUT, ":utf8");
	print OUTPUT "$source\n";
	print OUTPUT "$target\n";
	close(OUTPUT);

	`iconv -f UTF-8 /tmp/a > $param{output_directory}/$i`;
	
	exit if $n_selected++ > $param{n_select};
    }

    $i++;
}

close(SOURCE);
close(TARGET);
